from pathlib import Path

import scrapy


class QuotesSpider(scrapy.Spider):
    name = "quotes"

    async def start(self):

        for url in [f"https://quotes.toscrape.com/page/1/post" for i in range(1,6)]:
            yield scrapy.Request(url=url, dont_filter=True)

    def parse(self, response):

        page = response.url.split("/")[-2]
        filename = f"quotes-{page}.html"
        Path(filename).write_bytes(response.body)
        self.log(f"Saved file {filename}")
        datas = response.xpath('//div[@class="quote"]')
        for data in datas:
            author = data.xpath('.//small[@class="author"]/text()').get()
            yield {"author":author}
            nex = response.xpath('//li[@class="next"]/a/@href').get()
            if nex:
                yield response.follow(nex, callback=self.parse)


